package com.zdb.demo.spark.shuffle;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.scheduler.DAGScheduler;
import org.apache.spark.scheduler.TaskScheduler;
import scala.Tuple2;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

public class ShuffleTest {

    public static void main(String[] args) throws Exception {
        SparkConf conf = new SparkConf().setMaster("local[4]").setAppName("test");
        conf.set("spark.default.parallelism", "2");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        List<Long> list = new ArrayList<>();
        for (long i = 0L; i < 20L; i++) {
            list.add(i);
        }

        jsc
                .parallelize(list, 3)
                .mapToPair((Long aLong) -> {
            Thread.sleep(100);
            return new Tuple2<Long, Long>(aLong % 10, aLong);
        })
                .groupByKey()
                .foreach(new VoidFunction<Tuple2<Long, Iterable<Long>>>() {
            @Override public void call(Tuple2<Long, Iterable<Long>> t2)
                    throws Exception {
                long cnt = 0;
                for(Long l : t2._2) {
                    cnt ++;
                }
                System.out.println("--------:" + t2._1 + ",cnt:" + cnt);
            }
        });


        Thread.sleep(1000000* 10L);

        jsc.stop();
    }
}
